# -*- coding: utf-8 -*-
import scrapy

import json
import random
import logging
import requests
import urllib.parse
from retrying import retry
from redis import StrictRedis
from copy import copy, deepcopy
from FDC_spider.utils.redis_pool import POOL
from FDC_spider.constants import CHQ_STATUS_LI
from FDC_spider.items import FdcEstateGuidItem, FdcBuildingItem, FdcRoomItem

logger = logging.getLogger(__name__)


class ChongqingSpider(scrapy.Spider):
    name = 'chongqing'
    allowed_domains = ['cq315house.com']
    start_urls = ['https://www.cq315house.com/HtmlPage/PresaleDetail.html']
    project_li_url = 'https://www.cq315house.com/WebService/WebFormService.aspx/getParamDatas2'  # POST 项目列表url
    # start_urls = ['https://www.cq315house.com/HtmlPage/serviceSeaList.html']
    # project_li_url = 'https://www.cq315house.com/WebService/WebFormService.aspx/getParamDatas'  # POST 项目列表url
    building_url_temp = 'https://www.cq315house.com/HtmlPage/ShowRooms.html?buildingid={}&block={}'  # GET 楼栋url
    room_li_url = 'https://www.cq315house.com/WebService/WebFormService.aspx/GetRoomJson'  # POST 房号列表url
    status_url = 'https://www.cq315house.com/WebService/WebFormService.aspx/GetJsonStatus'  # POST 销售状态信息url
    status_li = list()
    custom_settings = {
        'CHQ_DUPS_EG': True,  # 开启项目（楼盘）增量爬取，默认开启
        'CHQ_DUPS_BD': True,  # 开启楼栋增量爬取，默认开启
    }

    def parse(self, response):
        """
        获取行政区列表
        :param response:
        :return:
        """
        option_li = response.xpath("//select[@id='selectxzq']/option[position()>1]")
        minrow, maxrow = (1, 11)
        for option in option_li:
            district_name = option.xpath("./text()").extract_first()
            district_value = option.xpath("./@value").extract_first()
            # 获取项目列表
            headers = {
                'Content-Type': 'application/json',
            }
            data = {
                'areaType': '',
                'entName': '',
                'location': '',
                'maxrow': str(maxrow),
                'minrow': str(minrow),
                'projectname': '',
                'siteid': district_value,
                'useType': '',
            }
            yield scrapy.Request(
                self.project_li_url,
                method='POST',
                headers=headers,
                body=json.dumps(data),
                callback=self.parse_project_li,
                meta=dict(district_name=district_name, district_value=district_value, page_num=1, ),
                dont_filter=True,
            )

    def parse_project_li(self, response):
        """
        获取项目列表
        :param response:
        :return:
        """
        district_name = copy(response.meta['district_name'])
        district_value = copy(response.meta['district_value'])
        page_num = copy(response.meta['page_num'])
        try:
            resp = json.loads(response.body.decode())
            project_li = json.loads(resp['d'])
            assert project_li
        except:
            logger.warning('{}-第{}页 项目列表获取为空'.format(district_name, page_num, ))
        else:
            redis_conn = StrictRedis(connection_pool=POOL)
            for project_dict in project_li:
                item_eg = FdcEstateGuidItem()
                item_eg['districtName'] = district_name
                item_eg['projectName'] = project_dict.get('projectname', None)
                item_eg['projectId'] = project_dict.get('projectid', None)
                item_eg['developerName'] = project_dict.get('enterprisename', None)
                item_eg['projectAddress'] = project_dict.get('location', None)
                item_eg['preSalePermit'] = project_dict.get('f_presale_cert', None)
                item_eg['preSaleBlockName'] = project_dict.get('blockname', None)
                # 利用redis集合特性，进行项目增量爬取
                eg_dups_dict = {'projectId': item_eg['projectId'], 'preSalePermit': item_eg['preSalePermit'], }
                if (self.settings['CHQ_DUPS_EG'] and redis_conn.sadd('chq_dups_eg', json.dumps(eg_dups_dict))) or (
                        not self.settings['CHQ_DUPS_EG']):
                    yield item_eg

                # 提取楼栋列表
                try:
                    bd_id_li = project_dict.get('buildingid', None).split(',')
                    bd_name_li = project_dict.get('blockname', None).split(',')[0:len(bd_id_li)]
                    assert len(bd_name_li) == len(bd_id_li)
                except:
                    logger.error('{}-{}-{} 楼栋名和楼栋id 数量不一致'.format(item_eg['districtName'], item_eg['projectName'],
                                                                  item_eg['preSalePermit']))
                else:
                    for bd_index, bd_name in enumerate(bd_name_li):
                        item_bd = FdcBuildingItem()
                        item_bd['districtName'] = item_eg['districtName']
                        item_bd['projectName'] = item_eg['projectName']
                        item_bd['projectId'] = item_eg['projectId']
                        item_bd['preSalePermit'] = item_eg['preSalePermit']
                        item_bd['blockName'] = bd_name
                        item_bd['buildingId'] = bd_id_li[bd_index]

                        # 获取房号列表
                        # 利用redis集合特性，添加增量爬取功能
                        if (self.settings['CHQ_DUPS_BD'] and redis_conn.sadd('chq_dups_bd', item_bd['buildingId'])) or (
                                not self.settings['CHQ_DUPS_BD']):
                            headers = {
                                'Content-Type': 'application/json',
                                'Referer': self.building_url_temp.format(item_bd['buildingId'],
                                                                         urllib.parse.quote(item_bd['blockName'])),
                            }
                            data = {
                                'buildingid': item_bd['buildingId'],
                            }
                            yield scrapy.Request(
                                self.room_li_url,
                                method='POST',
                                headers=headers,
                                body=json.dumps(data),
                                callback=self.parse_room_li,
                                meta=dict(item_bd=deepcopy(item_bd), ),
                                priority=10,
                            )
            # 关闭redis连接对象
            redis_conn.close()

            # 翻页
            next_page_headers = {
                'Content-Type': 'application/json',
                'Referer': self.start_urls[0],
            }
            page_num += 1
            maxrow = 10 * page_num + 1
            minrow = 10 * (page_num - 1) + 1
            next_page_data = {
                'areaType': '',
                'entName': '',
                'location': '',
                'maxrow': str(maxrow),
                'minrow': str(minrow),
                'projectname': '',
                'siteid': district_value,
                'useType': '',
            }
            yield scrapy.Request(
                self.project_li_url,
                headers=next_page_headers,
                method='POST',
                body=json.dumps(next_page_data),
                callback=self.parse_project_li,
                meta=dict(district_name=district_name, district_value=district_value, page_num=page_num, ),
                dont_filter=True,
            )

    def parse_room_li(self, response):
        """
        获取房号列表
        :param response:
        :return:
        """
        item_bd = copy(response.meta['item_bd'])
        floor_set = set()
        try:
            resp = json.loads(response.body.decode())
            resp_d = self.data_to_json(resp.get('d', None))
            room_li = list()
            for resp_dict in resp_d:
                resp_dict = self.data_to_json(resp_dict)
                resp_room_li = self.data_to_json(resp_dict.get('rooms', None))
                room_li += resp_room_li
            assert room_li
        except Exception as e:
            logger.error(
                '{}-{}-{}-{} 房号列表获取出错，error_msg:{}'.format(item_bd['districtName'], item_bd['projectName'],
                                                           item_bd['preSalePermit'], item_bd['blockName'], e))
        else:
            for room_dict in room_li:
                rn = room_dict.get('rn', None)
                item_rm = FdcRoomItem()
                item_rm['projectName'] = item_bd['projectName']
                item_rm['projectId'] = item_bd['projectId']
                item_rm['preSalePermit'] = item_bd['preSalePermit']
                item_rm['blockName'] = item_bd['blockName']
                item_rm['buildingId'] = item_bd['buildingId']
                item_rm['roomId'] = room_dict.get('id', None)
                item_rm['roomRegistrationNum'] = room_dict.get('fjh', None)
                item_rm['physicsFloor'] = room_dict.get('y', None)
                item_rm['nominalFloor'] = room_dict.get('flr', None)
                floor_set.add(item_rm['nominalFloor'])  # 用于提取楼栋总楼层
                item_rm['roomNo'] = self._get_room_no(item_rm['nominalFloor'], rn)
                item_rm['buildingArea'] = room_dict.get('bArea', None)
                item_rm['innerArea'] = room_dict.get('iArea', None)
                item_rm['apportionmentArea'] = room_dict.get('sArea', None)
                item_rm['roomLocation'] = room_dict.get('location', None)
                item_rm['unitNo'] = room_dict.get('unitnumber', None)
                item_rm['roomUse'] = room_dict.get('use', None)
                item_rm['roomStructure'] = room_dict.get('rType', None)
                item_rm['buildingStructure'] = room_dict.get('stru', None)
                item_rm['buildingAreaUnitPrice'] = room_dict.get('nsjmjg', None)
                item_rm['innerAreaUnitPrice'] = room_dict.get('nsjg', None)
                status_num = room_dict.get('status', None)
                item_rm['saleStatus'] = self._get_room_sale_status(status_num, item_rm)
                yield item_rm
        finally:
            # 提取楼栋总楼层
            try:
                assert floor_set
                floor_li = [int(i) for i in list(floor_set) if '负' not in i]
                assert floor_li
                floor_li.sort(reverse=True)
            except:
                item_bd['floorTotalNo'] = None
            else:
                item_bd['floorTotalNo'] = floor_li[0]
            finally:
                yield item_bd

    @staticmethod
    def data_to_json(data):
        """
        将data反序列化为json
        :return:
        """
        return json.loads(data) if isinstance(data, (str, bytes, bytearray)) else data

    @staticmethod
    def _get_room_no(flr: str, rn: str) -> str or None:
        """
        获取房号
        :return:
        """
        try:
            ret = str(flr) + '-' + str(rn)
        except:
            return
        else:
            return ret

    @retry(stop_max_attempt_number=3)
    def _get_json_status(self):
        """
        获取status列表
        :return:
        """
        headers = {
            'User-Agent': random.choice(self.settings['USER_AGENTS']),
            'Content-Type': 'application/json',
        }
        data = "{'para':''}"
        resp = requests.post(url=self.status_url, headers=headers, data=data, verify=False, timeout=10)
        assert resp.status_code == 200
        resp = json.loads(resp.content.decode())
        resp_d = self.data_to_json(resp['d'])
        resp_li = self.data_to_json(resp_d)
        return resp_li

    def _get_room_sale_status(self, sta, item):
        """
        解析并获取房号销售状态
        :param sta:
        :param item:
        :return:
        """
        try:
            sta = int(sta)
            if not self.status_li:
                try:
                    self.status_li = self._get_json_status()
                except:
                    self.status_li = CHQ_STATUS_LI
            for i in range(len(self.status_li) - 1, -1, -1):
                if (sta & self.status_li[i]['val']) == self.status_li[i]['val']:
                    if self.status_li[i]['name'] == '可售':
                        if (524288 & sta) == 524288:
                            is_pass = ((7518186 & sta) == 0) if ((524292 & sta) == 524292) else False
                            if not is_pass:
                                continue
                            else:
                                return self.status_li[i]['ab']
                        if (262144 & sta) == 262144:
                            is_pass = ((7516136 & sta) == 0) if ((262146 & sta) == 262146) else False
                            if not is_pass:
                                continue
                            else:
                                return self.status_li[i]['ab']
                    else:
                        return self.status_li[i]['ab']
        except Exception as e:
            logger.error(
                '{}/{}/{}/{}/{} 房号销售状态解析出错，error_msg:{}'.format(item['projectName'], item['preSalePermit'],
                                                                item['blockName'], item['unitNo'], item['roomNo'], e))
            return
